import json
import glob
import tqdm
import argparse

def convert_json_to_jsonl(input_pattern, output_file):
    with open(output_file, 'w') as outfile:
        for filename in tqdm.tqdm(glob.glob(input_pattern)):
            with open(filename, 'r') as infile:
                data = json.load(infile)
                for item in data:
                    json.dump(item, outfile)
                    outfile.write('\n')

def main(args):
    code_path = args.code_path
    convert_json_to_jsonl(f"{code_path}/pretokenized_*.json",
                            f"{code_path}/pretokenized.jsonl")

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--code-path", type=str, required=True)
    args = parser.parse_args()
    main(args)

